#!/usr/bin/env python
# encoding: utf-8
"""
@author: youfeng
@email: youfeng243@163.com
@license: Apache Licence
@file: www_zjcredit_gov_cn.py
@time: 2018/1/3 10:01
"""
import re
import sys

sys.path.append('..')
sys.path.append('../..')
from common import util
from common.mongo import MongDb
from logger import Logger

MONGO_DB_TARGET = {
    "host": "172.16.215.2",
    "port": 40042,
    "db": "crawl_data_new",
    "username": "work",
    "password": "haizhi",
}

# 日志模块
log = Logger('parse_www_zjcredit_gov_cn.log').get_logger()

target_db = MongDb(MONGO_DB_TARGET['host'], MONGO_DB_TARGET['port'], MONGO_DB_TARGET['db'],
                   MONGO_DB_TARGET['username'],
                   MONGO_DB_TARGET['password'], log=log)

MONGO_DB_SOURCE = {
    'host': '172.16.215.2',
    'port': 40042,
    'db': 'company_data',
    'username': 'work',
    'password': 'haizhi'
}

source_db = MongDb(MONGO_DB_SOURCE['host'], MONGO_DB_SOURCE['port'], MONGO_DB_SOURCE['db'],
                   MONGO_DB_SOURCE['username'],
                   MONGO_DB_SOURCE['password'], log=log)

WEB_PAGE_TABLE = 'web_page_www_zjcredit_gov_cn'
SEED_TABLE = 'new_zhejiang_company_list'


def parse_text(text):
    pattern = 'J\$(.*?)\$\$/corporation/eDetail.do\?id=.*?'
    regex = re.compile(pattern)
    search_list = regex.findall(text)

    result_list = []
    for company in search_list:
        result_list.append({
            '_id': company,
            '_in_time': util.get_now_time(),
            'crawl_status': 0,
        })

    return result_list


def main():
    result_list = []
    count = 0
    for item in target_db.traverse_batch(WEB_PAGE_TABLE, {}):
        text = item.get('text')

        count += 1
        parse_list = parse_text(text)
        result_list.extend(parse_list)
        if len(result_list) >= 500:
            source_db.insert_batch_data(SEED_TABLE, result_list)
            del result_list[:]

        log.info("解析进度: count = {}".format(count))

    if len(result_list) > 0:
        source_db.insert_batch_data(SEED_TABLE, result_list)
        del result_list[:]
    log.info("完成解析")


if __name__ == '__main__':
    main()
